{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/luca/anaconda3/envs/py27/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ordered-set package not found..\n"
     ]
    }
   ],
   "source": [
    "# Last run with python 2.7 and tensorflow cpu version (that's why there are some warning here and there...)\n",
    "# Previously was run with python 3.5 and tensorflow gpu verision\n",
    "\n",
    "from __future__ import absolute_import, print_function, division\n",
    "\n",
    "import far_ho as far\n",
    "import tensorflow as tf\n",
    "import far_ho.examples as far_ex\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sbn\n",
    "sbn.set_style('whitegrid')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "ss = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, shape=(None, 28**2), name='x')\n",
    "y = tf.placeholder(tf.float32, shape=(None, 10), name='y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /media/luca/DATA/Progs/FAR-HO/far_ho/examples/MNIST_DATA/train-images-idx3-ubyte.gz\n",
      "Extracting /media/luca/DATA/Progs/FAR-HO/far_ho/examples/MNIST_DATA/train-labels-idx1-ubyte.gz\n",
      "Extracting /media/luca/DATA/Progs/FAR-HO/far_ho/examples/MNIST_DATA/t10k-images-idx3-ubyte.gz\n",
      "Extracting /media/luca/DATA/Progs/FAR-HO/far_ho/examples/MNIST_DATA/t10k-labels-idx1-ubyte.gz\n",
      "datasets.redivide_data:, computed partitions numbers - [0, 7000, 14000, 70000] len all 70000 DONE\n"
     ]
    }
   ],
   "source": [
    "# load a small portion of mnist data\n",
    "datasets = far_ex.mnist(folder=os.path.join(os.getcwd(), 'MNIST_DATA'), partitions=(.1, .1,))\n",
    "datasets = far_ex.Datasets.from_list(datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ground model weights (parameters)\n",
      "<tf.Variable 'model/fully_connected/weights:0' shape=(784, 300) dtype=float32_ref>\n",
      "<tf.Variable 'model/fully_connected/biases:0' shape=(300,) dtype=float32_ref>\n",
      "<tf.Variable 'model/fully_connected_1/weights:0' shape=(300, 10) dtype=float32_ref>\n",
      "<tf.Variable 'model/fully_connected_1/biases:0' shape=(10,) dtype=float32_ref>\n",
      "Initial model weights (hyperparameters)\n",
      "<tf.Variable 'inital_weight_model/fully_connected/weights:0' shape=(784, 300) dtype=float32_ref>\n",
      "<tf.Variable 'inital_weight_model/fully_connected/biases:0' shape=(300,) dtype=float32_ref>\n",
      "<tf.Variable 'inital_weight_model/fully_connected_1/weights:0' shape=(300, 10) dtype=float32_ref>\n",
      "<tf.Variable 'inital_weight_model/fully_connected_1/biases:0' shape=(10,) dtype=float32_ref>\n"
     ]
    }
   ],
   "source": [
    "# build up a feddforward NN calssifier\n",
    "import tensorflow.contrib.layers as tcl\n",
    "with tf.variable_scope('model'):\n",
    "    h1 = tcl.fully_connected(x, 300)\n",
    "    out = tcl.fully_connected(h1, datasets.train.dim_target)\n",
    "    print('Ground model weights (parameters)')\n",
    "    [print(e) for e in tf.model_variables()];\n",
    "with tf.variable_scope('inital_weight_model'):\n",
    "    h1_hyp = tcl.fully_connected(x, 300,\n",
    "                                 variables_collections=far.HYPERPARAMETERS_COLLECTIONS, \n",
    "                                trainable=False)\n",
    "    out_hyp = tcl.fully_connected(h1_hyp, datasets.train.dim_target,\n",
    "                                 variables_collections=far.HYPERPARAMETERS_COLLECTIONS,\n",
    "                                 trainable=False)\n",
    "    print('Initial model weights (hyperparameters)')\n",
    "    [print(e) for e in far.utils.hyperparameters()];\n",
    "#     far.utils.remove_from_collection(far.GraphKeys.MODEL_VARIABLES, *far.utils.hyperparameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get an hyperparameter for weighting the examples for the inner objective loss (training error)\n",
    "weights = far.get_hyperparameter('ex_weights', tf.zeros(datasets.train.num_examples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-8-f0e5e37c4cb9>:4: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# build loss and accuracy \n",
    "# inner objective (training error), weighted mean of cross entropy errors (with sigmoid to be sure is > 0)\n",
    "with tf.name_scope('errors'):\n",
    "    tr_loss = tf.reduce_mean(tf.sigmoid(weights)*tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=out))\n",
    "    # outer objective (validation error) (not weighted)\n",
    "    val_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=out))\n",
    "accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(y, 1), tf.argmax(out, 1)), tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# optimizers\n",
    "# get an hyperparameter for the learning rate\n",
    "lr = far.get_hyperparameter('lr', 0.01)\n",
    "io_optim = far.GradientDescentOptimizer(lr)  # for training error minimization an optimizer from far_ho is needed\n",
    "oo_optim = tf.train.AdamOptimizer()  # for outer objective optimizer all optimizers from tf are valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hyperparameters to optimize\n",
      "<tf.Variable 'inital_weight_model/fully_connected/weights:0' shape=(784, 300) dtype=float32_ref>\n",
      "<tf.Variable 'inital_weight_model/fully_connected/biases:0' shape=(300,) dtype=float32_ref>\n",
      "<tf.Variable 'inital_weight_model/fully_connected_1/weights:0' shape=(300, 10) dtype=float32_ref>\n",
      "<tf.Variable 'inital_weight_model/fully_connected_1/biases:0' shape=(10,) dtype=float32_ref>\n",
      "<tf.Variable 'ex_weights:0' shape=(7000,) dtype=float32_ref>\n",
      "<tf.Variable 'lr:0' shape=() dtype=float32_ref>\n"
     ]
    }
   ],
   "source": [
    "print('hyperparameters to optimize')\n",
    "[print(h) for h in far.hyperparameters()];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build hyperparameter optimizer\n",
    "farho = far.HyperOptimizer()\n",
    "run = farho.minimize(val_loss, oo_optim, tr_loss, io_optim, \n",
    "                     init_dynamics_dict={v: h for v, h in zip(tf.model_variables(), far.utils.hyperparameters()[:4])})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variables (or tensors) that will store the values of the hypergradients\n",
      "Tensor(\"errors/Mean_1_1/gradients_3/errors/Mean_1_1/Vectorization_2/Reshape_1_grad/Reshape:0\", shape=(300,), dtype=float32)\n",
      "Tensor(\"errors/Mean_1_1/gradients_3/errors/Mean_1_1/Vectorization_2/Reshape_grad/Reshape:0\", shape=(784, 300), dtype=float32)\n",
      "<tf.Variable 'errors/Mean_1/ex_weights/hypergradient:0' shape=(7000,) dtype=float32_ref>\n",
      "Tensor(\"errors/Mean_1_1/gradients_3/errors/Mean_1_1/Vectorization_2/Reshape_2_grad/Reshape:0\", shape=(300, 10), dtype=float32)\n",
      "Tensor(\"errors/Mean_1_1/gradients_3/errors/Mean_1_1/Vectorization_2/Reshape_3_grad/Reshape:0\", shape=(10,), dtype=float32)\n",
      "<tf.Variable 'errors/Mean_1/lr/hypergradient:0' shape=() dtype=float32_ref>\n"
     ]
    }
   ],
   "source": [
    "print('Variables (or tensors) that will store the values of the hypergradients')\n",
    "print(*far.hypergradients(), sep='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy 0.10657143\n",
      "validation accuracy 0.10414286\n",
      "--------------------------------------------------\n",
      "training accuracy 0.49142858\n",
      "validation accuracy 0.463\n",
      "learning rate 0.011\n",
      "norm of examples weight 0.0786997\n",
      "--------------------------------------------------\n",
      "training accuracy 0.5412857\n",
      "validation accuracy 0.5167143\n",
      "learning rate 0.011991728\n",
      "norm of examples weight 0.15491235\n",
      "--------------------------------------------------\n",
      "training accuracy 0.651\n",
      "validation accuracy 0.62057143\n",
      "learning rate 0.012977126\n",
      "norm of examples weight 0.22778706\n",
      "--------------------------------------------------\n",
      "training accuracy 0.67071426\n",
      "validation accuracy 0.64185715\n",
      "learning rate 0.013939605\n",
      "norm of examples weight 0.30000913\n",
      "--------------------------------------------------\n",
      "training accuracy 0.6807143\n",
      "validation accuracy 0.65828574\n",
      "learning rate 0.014869263\n",
      "norm of examples weight 0.37109795\n",
      "--------------------------------------------------\n",
      "training accuracy 0.6892857\n",
      "validation accuracy 0.6668571\n",
      "learning rate 0.015759856\n",
      "norm of examples weight 0.44072804\n",
      "--------------------------------------------------\n",
      "training accuracy 0.6957143\n",
      "validation accuracy 0.677\n",
      "learning rate 0.01660772\n",
      "norm of examples weight 0.5086269\n",
      "--------------------------------------------------\n",
      "training accuracy 0.7015714\n",
      "validation accuracy 0.685\n",
      "learning rate 0.01741094\n",
      "norm of examples weight 0.5745802\n",
      "--------------------------------------------------\n",
      "training accuracy 0.70357144\n",
      "validation accuracy 0.6915714\n",
      "learning rate 0.018168906\n",
      "norm of examples weight 0.6384008\n",
      "--------------------------------------------------\n",
      "training accuracy 0.7074286\n",
      "validation accuracy 0.6965714\n",
      "learning rate 0.018881945\n",
      "norm of examples weight 0.69995666\n",
      "--------------------------------------------------\n",
      "training accuracy 0.7095714\n",
      "validation accuracy 0.7007143\n",
      "learning rate 0.01955109\n",
      "norm of examples weight 0.7591762\n",
      "--------------------------------------------------\n",
      "training accuracy 0.71085715\n",
      "validation accuracy 0.7038571\n",
      "learning rate 0.0201779\n",
      "norm of examples weight 0.8160307\n",
      "--------------------------------------------------\n",
      "training accuracy 0.7131429\n",
      "validation accuracy 0.7052857\n",
      "learning rate 0.02076423\n",
      "norm of examples weight 0.8705156\n",
      "--------------------------------------------------\n",
      "training accuracy 0.714\n",
      "validation accuracy 0.7082857\n",
      "learning rate 0.021312008\n",
      "norm of examples weight 0.9226469\n",
      "--------------------------------------------------\n",
      "training accuracy 0.7164286\n",
      "validation accuracy 0.70942855\n",
      "learning rate 0.021823296\n",
      "norm of examples weight 0.9724517\n",
      "--------------------------------------------------\n",
      "training accuracy 0.718\n",
      "validation accuracy 0.712\n",
      "learning rate 0.022300176\n",
      "norm of examples weight 1.0199825\n",
      "--------------------------------------------------\n",
      "training accuracy 0.719\n",
      "validation accuracy 0.71485716\n",
      "learning rate 0.02274472\n",
      "norm of examples weight 1.0653107\n",
      "--------------------------------------------------\n",
      "training accuracy 0.72057146\n",
      "validation accuracy 0.71685714\n",
      "learning rate 0.023158928\n",
      "norm of examples weight 1.1085085\n",
      "--------------------------------------------------\n",
      "training accuracy 0.7227143\n",
      "validation accuracy 0.71814287\n",
      "learning rate 0.023544773\n",
      "norm of examples weight 1.149658\n",
      "--------------------------------------------------\n",
      "training accuracy 0.72414285\n",
      "validation accuracy 0.72014284\n",
      "learning rate 0.023904106\n",
      "norm of examples weight 1.1888547\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# run hyperparameter optimization \n",
    "T = 200 # performs 200 iteraitons of gradient descent on the training error (rise this values for better performances)\n",
    "# get data suppliers (could also be stochastic for SGD)\n",
    "tr_supplier = datasets.train.create_supplier(x, y)\n",
    "val_supplier = datasets.validation.create_supplier(x, y)\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "print('training accuracy', accuracy.eval(tr_supplier()))\n",
    "print('validation accuracy', accuracy.eval(val_supplier()))\n",
    "print('-'*50)\n",
    "\n",
    "tr_accs, val_accs = [], []\n",
    "for _ in range(20):\n",
    "    run(T, inner_objective_feed_dicts=tr_supplier, outer_objective_feed_dicts=val_supplier)\n",
    "    tr_accs.append(accuracy.eval(tr_supplier())), val_accs.append(accuracy.eval(val_supplier()))\n",
    "    print('training accuracy', tr_accs[-1])\n",
    "    print('validation accuracy', val_accs[-1])\n",
    "    print('learning rate', lr.eval())\n",
    "    print('norm of examples weight', tf.norm(weights).eval())\n",
    "    print('-'*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fe99fcc0890>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD1CAYAAABEDd6nAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPW9//FX9sm+kBAIBNm/hEV2ULFeihWwbEoFEYtYW1ttqcrt7/ZqN73Ua+2mtS53KSK4FEVBhV4oxa24CwGUZfgqOyEr2SbbTDKZ8/vjTJIhmcBkncnk83w85jEzZ75n5pPD8M7J93zP94QYhoEQQojgEervAoQQQnQuCXYhhAgyEuxCCBFkJNiFECLISLALIUSQkWAXQoggE+7vAgCys7NlzKUQQrTD5MmTQ5ovC4hgB5g8eXK71rNarWRlZXVyNZ1H6usYqa9jpL6OC+Qas7OzvS6XrhghhAgyEuxCCBFkJNiFECLISLALIUSQkWAXQoggI8EuhBBBRoJdCCH8pKumTQ+YcexCCNETuVwGFQ4n5dV1lNc03cpqapueN3+tug5bTR0hIfB/93yNzJSYTq1Jgl0IIbyodboorLBjLbRzuj6PApuDApudApuDwgo7BTY7hRUObDV1uC6y4x0ZFkpiTASJ0REkRUfQL8GCSo8nITqCgcnRpMVHdXrtEuxCiF7DWe+irKaOsupaSqvrKKmqpbDCQaHN3hjaDYFdUlXrsWYuAOGhIfSNj6JvgoUhqbFMG5JCckwkidERF9ySPJZZIkIJCWlx1n+XkmAXQvQ4tU4X1bVOKuxOymvqKHUHdVl1LaVV5vMyz2XV5rIKu9Pr+4WGQFp8FOkJFgYmxzDpsmTS4y2kJ0ThKC9iypjhpCdYSImJJDTUh5Cuq4HKfMgvgMqGW6G5rLKw6bnLCd/dBcmXder2kWAXQnQLl8vAZq9rDNmy6lqsxyvYW36aaoeTqtr6pvtaJ1WOeqocTvNxs9fq6i9+0DE+Kpyk2AiSYyJJiolkcGqs+3HEBffJMZGkJ0TRJy6KMG+BbRgcPXSeUXHVUJ0HpTawl3vcyqCyyCO83YHtsHmpKgRi0yA+HeLSoe9oSBkCcX07ZwN7kGAXQrSZYRjYapzk2WooqaqlrDGs6yit8txTbnqtvNW+6KLGR7GRYcREhZv3keHERoWRFBPJgGT381ZeN0Pa7AJJiokgIszLgD+nw2PPuQBsBZBb1BTSjoaw9ghvh41RLu97+Y2iEsxwjkuHfuMgrl/T87j0pscxfSCseyJXgl0I0UJNbT255TXkldnJLa8ht6zZ43I71bX1XteNjghrDNnk2Aj6J0WTHNO095zssddcnHeW8aMVsVFhWMLDfOvm8ORyQU0pVOaZYZ3v0c1RkX9hkNvLvL9HZBxYEs2AtiSaIZw6onFZgc1B+qARYHG/HpVo3lsSzWUR0W3cul1Pgl2IIFbvMqiw11FTV4+91kV1nZOa2npq6uqpqa2n0uEkr9xOblkNuWV28tzBXVpd1+K90uKjyEi0MDI9nn8Z2ZeMJAv9Ei2kxEY2dmskxURgiQjzuT5rVf6lR4U4KqH0FJSeNO9LTjY9LzsLrpa1Eh7t7vLoB2kKhlzTtPcc77FHHZsGYREX/fgSq5X0AJ22tzUS7EL0AJUOJ/nlNeSXO8grryG/3E6+zU5RhYNqj6BuuK+udWKvc1Fb7wJOXvL9EyzhZCRF0z/RwoTMJDKSoslIstA/MZqMxGjSE6OICvc9sNvEMMy964aw9gzu0lNQVXRhe0sSJA+G/uMhayHE9/cIbHd4R8ZBN49ECSQS7EL4kWEYlFbXXRDW+eV28srN4Xd55ebzSkfLft7kmAjS4qOIjQonJtLs/rBEhBETGUZ0RBjRkeFUlpdwWUY/LO5lTa+Z97FR4fRLtBAX1YVR4HKZXSHlZ6HsjPv+LJSfZWjBV7A5H5w1Te1DQiFhoDlSRF1vhnjyEPNAY/JgiE7uulqDhAS7EB1gGAZVDic2ex22GicV9jps9joq7E5sNXXY7E2veS4325lnK5p71U1CQ6BvvIX0RAvD0+K4engq/RIt9E+0kJ7QdO9Ll4d59Z+hXfXjm+rrwHauMazN+zNmiJedNV+rr71wHUsSJGVSG59J1Jh5F4Z3YiaER3ZtzUFOgl0IL1wug5Lq2saTVQo9Tl7xPPOwqMKBy7h4V0dkeCgJlggSLOHER5v3A5KiSYgOJ8F9JmK/BLO/ul+ihbS4KMK9jeroCvVOqK0w+7EdFR43W9Pj2sqWyxpuNWXm2Gzjwl9OxPWDpEzImAijF5phnTTIfZ8JUfEA5ATwZed6Mgl20StV1zo5ml/B8cJKCisaAtsd2u4wd3oZm9cnNpK+CeaJK1n9EsBRwbDMfiRYIoi3RJAQHW7eW8z7eEt4mw4mdonaanff9QnzVnzc/fikuTeNDxNRRcaZt6j4pltsmjkyJGGAGdYN4Z0wACIsXf5jidZJsIugZhgGOaU1HM2vwJpnw5pn42h+BaeKq/CcWC8xOoL0BPPMw2FpqY2P0xOi3EFu7klHhl+4J212dQzr5p/KC0ell/A+yfBCDTXNDj7GpELKUBh8tdmPbUm6MLCjEiDKI8Qj4yDUz7+cRJtIsIugUV3rROdXXBjieRVUeBx4vKxPDFn9Elg0IYOs/gmMTI+nf6Jv/dXdxlXfdFZjTanZ3dHisft5dbG5512Zf+F7xKZByjCq0qeSNGQS9BlqhnnyEIhO8s/PJbqNT8GulJoLPAGEAWu11o82e/1x4OvupzFAX611kvu1lcAv3K89rLXe0BmFi97H5TIora6lqNJBUYV5O1dagzXfDPCTHnvhsZFhjOqfwKKJGYzql0BW/wRG9YsntitHf1yKYUDVeSg+1nQrOw3VJR5hXeY+Hf0i3SPhFnMvOzrJPJtx+DfMg44pQ6HPMDO8LQkA5FmtJEkfdq9zyW+5UioMeBq4DsgB9iiltmqtjzS00Vqv9mj/Y2Ci+3EK8CAwBfObmu1et7RTfwrRo9XU1lNYYW8M64bgLrQ5OJlfTM1bxRRVODhf6b3fe1BKDFn941k4wQzx0f0TGJgc3fazGDuLo8LsCik+5nHvfuwob2oXGmH2Scf0Mcdfp41qCuyG++jklssC8ExHEVh82X2ZBhzTWp8AUEq9DCwCjrTS/hbMMAeYA+zSWpe4190FzAU2dqRo0XMZhsHR/Ap2f1nE+1+d5/OzZRd0lTQIDYE+cVEkRBhkpsUyql88afFRpMVH0Tfe0vg4PSGKmEg/7IUbBpSfJe7cbijZdWGIX9AtEmIeVOwzDC5fCn2Gu2/DzOXdNHeI6F18+VYNAM56PM8BpntrqJS6DBgCvHORdQe0vUzRk5VU1fL+V0Xs/vI8739VRGGFA4CR6XHcMHEA/ZPMA5N93Qco0+KjSImNJCw0xH1w0s9dCS6XeWAy7wDkfd50qykls6FNTB8zsIdfa4Z2Q4CnDJU9bNHtfAl2b3/PttYBuAx4TWvdMDuQz+tarVYfSmnJbre3e93u0Bvrc7oMrIV29uXWkJ1bw7FiBwYQFxnKpIxolo+LZ1JGDGmxDV+/OvNWXwnlcL4czndhfRflqiey4gyW0qNYSrV5K/uSsLoqAIzQcOyJw7D3/xr2ZIUtZjD0GY4rKrHle5UAJae6r3YveuP3r7P1hBqb8yXYc6BpxwQYSMPlRFpaBvyo2bozm637nrcV27tXFhB7dBfRW+o7U1zNP78qYveXRXx8vJhKh5Ow0BAmZiaxetJlfG1EKpcPTPI+53U31OdVvROKrJDrsSdecAjqqs3Xwy2QPhbGL4OMCdB/PCFpWUSHR9KwD17WS/59u0qg1weBXWN2drbX5b4E+x5ghFJqCHAOM7yXN2+klFJAMvCxx+KdwCNKqYbJHWYDD/hetghU9rp6Pj5RzLtHC9n9ZRGnis0wHJAUzYLxGfzLyFSuHJZKYvTFZ87rVrXVcG4vnPkETn8EOXvMsyrBHKvd73KYtNKcXKr/eEgdKX3goke65LdWa+1USq3CDOkwYJ3W+rBSag2wV2u91d30FuBlrbXhsW6JUurXmL8cANY0HEgVPU9xpYN3jhbylrWA9786T3VtPdERYVwxNIWVVw3mmpFpDE2N7fbrO7aqugTOfGzeTn9s9pG7nEAIpI8x98QzrzBPe08ZCqHddBq/EF3Mp90RrfV2YHuzZb9q9vyhVtZdB6xrZ33CjwzD4HhRJbuOmGG+70wphgH9EiwsnjSAa7PSuXJon8A4uccwzEmnPIP8vDZfC4uEAZPhqh/DoKsgc6rMECiCmvydKS7grHex51Qpb1kLeNta0NjFMnZAAvdeO4JvZKUzJiMhMPbKS07AsbebgrzCfegnKhEGTYfxN5tBnjFR5i4RvYoEu8Bmr+Ofuoi3rQW8q4sor6kjMiyUq4b34btfG8q1o/qSkRQAQ/YMA3L3g94OR/8PCt2nUsRnwGVXwiD3rW+WzG0iejUJ9l6qpraev32Ry18/zONgwUmcLoOU2Ei+kZXOdaP7cvWItK69+IKv6uvg+DtmkOsd5myEIaFw2QyY8xtQc81T6APhLwghAkQA/M8V3elYYQUvfXqGzdk52OxOBiRE8N2vDeG6rHQmDkpu83DELmG3wbFdcHQ7I/UOqKuCiBgYNgtm/RJGzoGYFH9XKUTAkmDvBRzOev5+KJ+XPj3DZydLiAgLYe7Y/tw6fRAJ9gJGjw6AMbq2vKYulpO7zQsUx6RSMXAWSVfcCkNnyhmcQvhIgj2InTpfxcbPzvBqdg4lVbUMSonh/utHcdPkgaTGmVeGt1oL/VdgyUk4tNkM9HPuEy1ShsIVd8Go+TBwKnn6S5JUAPziEaIHkWAPMnX1Lt46UsBLn57hg2PnCQsN4bqsdG69YhAzhqX6b8bDBo5KsG6F/S/B6Q/MZQMmw7W/AjUP0pT0lwvRQRLsQSKntJpX9pzl5T1nKapwkJFo4SfXjWTp1EzSE/w81M8wzCGJ+1+CI2+YZ3umDDX7y8cvg8SB/q1PiCAjwd6D1bsM3tOFvPTpGd7VZpfK11Vfbp0+iJmqr/8PhJbnwIGNcOAlc3bEyDgYcyNM/DZkTpc9cyG6iAR7D+Rw1vP6vnP8z+4TnDxfRVp8FKu+Ppybp2YyMDnGv8XV1ZgHQPe/CCfeAwwY/DX4l383r1YfGevf+oToBSTYe5Aqh5ONn53hL++foMDmYOyABJ5aPpE5Y/oREebHeU4Mwzz4uf9FOLTFvEpQ4iAzzCfcAsmD/VebEL2QBHsPUFpVy/qPTrHh41OUVddx5dA+/GHJeK4enurfU/uriuHAi2bf+XkN4dEwehFMWG7upcukWkL4hQR7AMsrr+Evu0+y8bMz1NTVc93odO6eOYxJg/w8gZUtDz5+CvauM+cuz5wOC/5s9p+7L6IshPAfCfYAdKKokv/+53Fe338OlwGLxmdw18xhjEyP929hJSfhwyfMg6Guehi3BK6+z5ybRQgRMCTYA8ihc+U8894xdhzKJzIslFumDeLOrw0lM8XPB0QLj8IHj8HB18zJtSZ+G666B1KG+LcuIYRXEux+ZhgGn5wo4Zn3jvH+V+eJjwrn7n8ZxndmDCEtPsq/xeXuh/f/CNZt5lwtV9wNV66ChP7+rUsIcVES7H70RU4ZD249zP4zZaTGRfLTuYpvX3EZCRY/X07u1IdmoB9/GyyJcM1PYfpdENvHv3UJIXwiwe4nH3x1nu+/sJcESwS/XjSGJVMy/XslIsMwL1rx/h/Ms0RjUuHaB2Hq9+SAqBA9jAS7H/z9UB73bDzA0LRYnr9jGn39ecq/4YIjb5p76HmfQ8IAuP53MHEFRPq5b18I0S4S7N1s096z3L/5C8ZnJvHc7VNJion0TyGOSjj4KkN3/wlsp8y5WxY+CZcvg3A/1SSE6BQS7N1o7fsnePj/rHxtRCr/s2IyMZF+2Pz5B83x51+8CrUVGEkj4FvPmmPQ5XJyQgQFCfZuYBgGj+36kiffOcb1Y/vxp2UTiArvxhCtq4HDr5uBnrMHwi1mkE+5g5MVcWSNHt19tQghupwEexdzGQYPbT3Mho9Ps3TKQB65cRzh3TWvS5GGvc/B538Fezn0GWFeJ3T8sqZLy1mt3VOLEKLbSLB3obp6F3/4oIh3T1Ry59eG8LNvZnX93C5OhznufO86OP0hhEaYsypOucO8ALRMlStE0JNg7yL2unp+9NI+3j1Ryb/NUfxw5rCuDfXi45C93jzdv7rYnFHxG/8BE26FuLSu+1whRMCRYO8CFfY6vrdhL5+dKuFH01P50deHd80H1TvN64XuXQcn3oWQMBj1TZj8HRj6dZldUYheSoK9kxVXOlj53GcczavgTzdPYGSUrWs+yJYHr94OZz+BhIHw9Z+bY8/ldH8hej0J9k6UW1bDimc/Jae0hv+9bTKzRqVjtXZBsJ/6AF79jnnt0Bv+Cy6/WYYqCiEaSbB3khNFlax49jNsNXU8f8c0pg/tgnlVDAM+ehLeesg8oWjlVpkyVwjRgk/BrpSaCzwBhAFrtdaPemmzFHgIMIDPtdbL3cvrgYPuZme01gs7oe6Acji3nJXrPsMwYOP3r2DsgMTO/xC7Dd78oTniJWshLHpa5nARQnh1yWBXSoUBTwPXATnAHqXUVq31EY82I4AHgBla61KlVF+Pt6jRWk/o5LoDxp5TJdzx3B7iLeG88L3pDEuL6/wPKbTCK982L3Qx+2Fz6lwZtiiEaIUve+zTgGNa6xMASqmXgUXAEY82dwJPa61LAbTWhZ1daCDac6qEFc9+SkZiNC98bzoDkqI7/0MOvgZbfwxR8bByGwye0fmfIYQIKr4E+wDgrMfzHGB6szYjAZRSH2J21zyktf67+zWLUmov4AQe1Vq/0bGSA8dfPz1DTGQ4m+66ktS4Tr4ohrMW/vFz+Ox/YdCVsGQ9xPfr3M8QQgQlX4Ld29/8hpf3GQHMBAYC7yulxmqty4BBWutcpdRQ4B2l1EGt9fHmb2ht56ntdru93et21L6ThQxPjqDo7AmKWmnTnvrCqwsZ8NHPiSk+SPHIZRSOXwU5pUBph2vujPq6k9TXMVJfx/WEGpvzJdhzgEyP5wOBXC9tPtFa1wEnlVIaM+j3aK1zAbTWJ5RS7wETgRbBnpXVvtEdVqu13et2RE1tPWfLT7Bw0mVkZalW27W5vhP/hG13gNMOS9bTZ8yNdOV1i/y1/Xwl9XWM1NdxgVxjdna21+W+nJq4BxihlBqilIoElgFbm7V5A/g6gFIqFbNr5oRSKlkpFeWxfAYX9s33WEfzbbgMGJPRSSNgDAM+eBxeuAFi+sCd75gzMAohRBtdMti11k5gFbATsAKbtNaHlVJrlFINQxd3AsVKqSPAu8C/aa2LgSxgr1Lqc/fyRz1H0/Rkh3LNE4/GZHTCkEN7Obx8qzk+ffQiM9TTWv8rQAghLsancexa6+3A9mbLfuXx2AD+1X3zbPMRMK7jZQaeI7nlJEZHMDC5gyNh8g/BphVQdsacUveKu2UooxCiQ+TM03Y6nGtjTEZCx2ZstG6DzXeCJRFW/g0uu7LzChRC9Foy/V871NW7OJpX0bFumCNvwqaV0G8s/GC3hLoQotPIHns7HCuspLbe1f6pA6zb4LU7YOAU+PZm8+QjIYToJLLH3g6HO3Lg1Po3c7rdjElw62sS6kKITifB3g6HzpUTHRHGkNQ2zgtz9P/g1ZWQMdHcU5dJvIQQXUCCvR2O5NrI6h9PWGgbDpwe3W72qfefIKEuhOhSEuxt5HIZHM4tb9uJSXoHbLoN+l8OK7aYo2CEEKKLSLC30emSaqpq6xk7wLc97rjcD+CVFdBvHHxbQl0I0fUk2NvocG454ONUAl/+gwEfPmAOaVzxOkQndXF1Qgghwd5mh87ZCA8NYUT6JQ6cfrULXrkVR+IwCXUhRLeSYG+jw7nljEyPJyr8IheP/uotc+6XvlmcmflniE7uvgKFEL2eBHsbGIbROJVAq469BS8vNyfxWvEGrkgZ/SKE6F4S7G2Qb7NTUlXb+hmnx96GjcshbSTc9ibEpHRvgUIIgQR7mxw+d5EzTo+/Y+6pp46E27ZKqAsh/EaCvQ0O5ZYTEgJZ/ZsF+/F3YeMt0Ge47KkLIfxOgr0NDufaGJIaS2yUx9xpJ96DjcsgZZi5px7blReyE0KIS5Ngb4MjubYLx6+f3QN/XQYpQ2GlhLoQIjBIsPuotKqWc2U1jPXsX9+zFiKi3Xvqqf4rTgghPEiw+6hpql6PPfbc/ZA5HeLS/FSVEEK0JMHuo0ONUwm499gdFXD+S3MKXiGECCAS7D46nGtjQFI0ybGR5oK8zwFDgl0IEXAk2H10OLec0Z7967n7zXsJdiFEgJFg90GVw8nJ81WMbd6/npgp/etCiIAjwe4Da54Nw2h2xmnufsiY4L+ihBCiFRLsPmgcEdNwcY2aUig5Id0wQoiAJMHug0PnyukTG0m/BIu5IO9z816CXQgRgCTYfXA418bojARCQtwXrz63z7zvL10xQojAI8F+CQ5nPV8WVLQ8MSl5iEz2JYQISBLsl/BVQSVOl3HhxatzD0g3jBAiYIVfugkopeYCTwBhwFqt9aNe2iwFHgIM4HOt9XL38pXAL9zNHtZab+iEurtNi4tXV52H8jMw7U4/ViWEEK275B67UioMeBq4HhgN3KKUGt2szQjgAWCG1noMcJ97eQrwIDAdmAY8qJTqURcAPXTORlxUOJelxJgLcg+Y97LHLoQIUL50xUwDjmmtT2ita4GXgUXN2twJPK21LgXQWhe6l88BdmmtS9yv7QLmdk7p3eNwbjmj+ycQGuo+cNpwxmn/8f4rSgghLsKXYB8AnPV4nuNe5mkkMFIp9aFS6hN3142v6wasepeBNa+i2VQC+6DPCLDIRaqFEIHJlz72EC/LDC/vMwKYCQwE3ldKjfVxXQCsVqsPpbRkt9vbve6lnCmrpaaunpSQqsbPGH5mD9V9J5Pr42d2ZX2dQerrGKmvYwK9PugZNTbnS7DnAJkezwcCuV7afKK1rgNOKqU0ZtDnYIa957rvefuQrKws3ypuxmq1tnvdS/nywDkgh+umjDKvc2rLg5oiErNmkujjZ3ZlfZ1B6usYqa9jAr0+COwas7OzvS73Jdj3ACOUUkOAc8AyYHmzNm8AtwDrlVKpmF0zJ4DjwCMeB0xnYx5k7REOnSsnMjyU4X3jzAV5cuBUCBH4LtnHrrV2AquAnYAV2KS1PqyUWqOUWuhuthMoVkodAd4F/k1rXay1LgF+jfnLYQ+wxr2sRzica2NUv3giwtybKXc/hIRCv3H+LUwIIS7Cp3HsWuvtwPZmy37l8dgA/tV9a77uOmBdx8rsfoZhcDjXxjfH9WtamLsf0kZBZKz/ChNCiEuQM09bkVNaQ3lNXdOJSYZhzhEj3TBCiAAnwd6KpotXu4c1ludA9XkJdiFEwJNgb8Xh3HLCQkPM0TDgcSm8Sf4rSgghfCDB3orDuTaGpcViiQgzF+Tuh9BwSB/j38KEEOISJNhbcTi3vOVUvX1HQ4TFf0UJIYQPJNi9KKpwUGBzNPWvG4b7GqfSvy6ECHwS7F60mKq39CTYyyTYhRA9ggS7Fw0jYhon/2o4cDpADpwKIQKfBLsXh3PLGZQSQ2J0hLkgdz+ERUFaYM4XIYQQniTYvTica2vqXwfz4hr9xkJ4pP+KEkIIH0mwN2Oz13G6uJqxA9z96y6XXONUCNGjSLA3c6R5/3rxMaitkGAXQvQYEuzNtJhKoPGMUwl2IUTPIMHezOFz5fSNj6JvvPtEpNz9EBEDqcq/hQkhhI8k2JtpeeB0P/S7HMJ8muFYCCH8ToLdg72unmNFlU0HTuudkP+FdMMIIXoUCXYPR/MrqHcZTXvs57+EumoJdiFEjyLB7qHFVAK5+8x7CXYhRA8iwe7h0DkbCZZwBiZHmwty90NkPPQZ7t/ChBCiDSTYPRxxT9UbEhJiLsjdDxkTIFQ2kxCi55DEcqurd2HNr2DsAHf/urMW8g+ZwS6EED2IBLvb8aJKap2upv71IivUO6R/XQjR40iwux0+J2ecCiGCgwS726HcciwRoQxNizMXnNsHliRIHuLfwoQQoo0k2N0O59rI6p9AWKjngdOJ0HAgVQgheggJdsDlMrDm2hjb0L9eZ4fCI9INI4TokSTYgTMl1VQ4nE396wWHweWUYBdC9EgS7HhO1StnnAohej4JdsypBMJDQxjZz33gNPcAxKRC4kD/FiaEEO3g01y0Sqm5wBNAGLBWa/1os9dvB34PnHMvekprvdb9Wj1w0L38jNZ6YSfU3akO5doYkR5PVHiYuSB3nxw4FUL0WJcMdqVUGPA0cB2QA+xRSm3VWh9p1vQVrfUqL29Ro7UO2NM3DcPg8LlyZo3qay6orYKio5C1wL+FCSFEO/nSFTMNOKa1PqG1rgVeBhZ1bVndp8DmoLiqtunAaf5BMFzSvy6E6LF86YoZAJz1eJ4DTPfS7ltKqWuAL4HVWuuGdSxKqb2AE3hUa/1GRwrubI1T9TZcXKPhjNP+AftHhhBCXJQvwe6to9lo9nwbsFFr7VBK3QVsAGa5Xxuktc5VSg0F3lFKHdRaH2/+hlartS11N7Lb7e1eF+C9z0sJAUJteVitBWRY3yMmOo1j58rgXFm737ez6utqUl/HSH0dE+j1Qc+osTlfgj0HyPR4PhDI9WygtS72ePoX4Lcer+W6708opd4DJgItgj0rK8vnoj1ZrdZ2rwtQsGcvQ1JjmXz5GHPB2ydg0NQOvWdn1tfVpL6Okfo6JtDrg8CuMTs72+tyX/rY9wAjlFJDlFKRwDJgq2cDpVR/j6cLAat7ebJSKsr9OBWYATQ/6OpXh3NtjG7oX7fb4PxX0r8uhOjRLrnHrrV2KqVWATsxhzuu01ofVkqtAfZqrbcC9yilFmL2o5cAt7tXzwL+Rynlwvwl8qiX0TR+U1Zdy7myGlZceZm5IO9zwICMSX6tSwghOsKncexa6+3A9mbLfuXx+AHgAS/rfQSM62CNXabpjNPmU/XKgVMhRM/Vq888bXnx6v2QOAhiU/1YlRBCdEyvDvYdh/IZlhZLSmykuaAQi1LvAAAZiUlEQVThGqdCCNGD9dpg/yKnjP1nyrh1urt/vaYUSk/KgVMhRI/Xa4N9w0eniYkM46Yp7om+5FJ4Qogg0SuDvbjSwbYvcvnWpIEkWCLMhXLgVAgRJHplsL+85yy1The3NQxzBDPYU4ZCdLL/ChNCiE7Q64LdWe/ixU9OM2N4H0akxze9kHtAumGEEEGh1wX7riMF5JXbWXnl4KaFlUVQflaCXQgRFHpdsK//6BQDkqK5Niu9aWHeAfNegl0IEQR6VbAfzbfx6ckSVlx5GWGhHpNWntsHhED/8X6rTQghOkuvCvYNH50mKjyUm6dkXvhC7n5IHQlR8d5XFEKIHqTXBHt5dR1v7D/HogkZJDecadogd790wwghgkavCfZXs89SU1fPyqsGX/iCLQ8q8yXYhRBBo1cEe73L4PmPTzN1cHLThF8N5IxTIUSQ6RXB/p4u5ExJNbd5DnFskLsfQsKgX8DOLiyEEG3SK4J9w8enSU+IYu7Yfi1fzN0HfbMgMqb7CxNCiC4Q9MF+vKiS3V8Wcev0y4gIa/bjGoZM1SuECDpBH+wvfHyaiLAQlk3LbPli+VmoLpb+dSFEUAnqYK90OHktO4d54/rTN97SsoEcOBVCBKGgDvYt+3KodDhbDnFscPYzCI2AvmO6tS4hhOhKQRvshmGw4aNTXD4wkQmZSS0b1Dvh4KswbBZEeNmbF0KIHipog/3DY8UcL6pi5ZWDCQkJadngq51QWQCTV3Z/cUII0YWCNtjXf3SKPrGRzLu8v/cG2Rsgrh+MmNO9hQkhRBcLymA/W1LN20cLWDYtE0tEWMsG5efg2C6YeCuEhXd/gUII0YWCMthf/OQ0oSEh3Dr9Mu8N9r8IhgsmrujewoQQohsEXbDX1Nbz8p6zzB6dTkZSdMsGrnrY/wIMnQkpQ7q7PCGE6HJBF+xbPz9HeU1d60Mcj79rnpg0SQ6aCiGCU1AFu2EYrP/oNKP6xTN9SIr3RvvWQ3QKjJrXrbUJIUR3Capg33OqFGuejdtaG+JYWQh6B0xYDuFR3V+gEEJ0A5+GhCil5gJPAGHAWq31o81evx34PXDOvegprfVa92srgV+4lz+std7QCXV7teHjUyRYwrlhYob3Bgf+Ci4nTLqtq0oQQgi/u2SwK6XCgKeB64AcYI9SaqvW+kizpq9orVc1WzcFeBCYAhhAtnvd0k6p3kN+uZ2/H8rnjhmDiYn08mMZBux7HgZdCWmqsz9eCCEChi9dMdOAY1rrE1rrWuBlYJGP7z8H2KW1LnGH+S5gbvtKvbiXPj2NyzBYccVg7w1OfQAlx+WgqRAi6PnSFTMAOOvxPAeY7qXdt5RS1wBfAqu11mdbWXeAtw+xWq0+Fdyc3W7n80NHeOGjM0wdEENV4WmshS3bZXz8Z+Ii4vgqbBRGOz+rvfW192frDlJfx3RFfYZhUF9fj2EYnfJeX3zxRSdU1TUCvT4IjBpDQkIICwvzfuzQC1+C3ds7Nf/GbQM2aq0dSqm7gA3ALB/XBSArK8uHUlqyWq0ctcdTZq9n1eyxZI1Ma9mougRe+ydMuo1R47p3il6r1drun607SH0d0xX1nTx5kvj4ePr06ePzf+TW1NTUEB3t5XyOABHo9YH/azQMg+LiYioqKhgy5MJzb7Kzs72u40tXTA7geZWKgUCuZwOtdbHW2uF++hdgsq/rdob1H51maGosVw9P9d7gi1eg3iEHTUWPYLfbOyXURXAICQmhT58+2O12n9fxJdj3ACOUUkOUUpHAMmCrZwOllOdMWwuBhr9NdwKzlVLJSqlkYLZ7WafRRXY+P1vGbVdeRmiol/8IDQdNMyZC/8s786OF6DIS6sJTW78Plwx2rbUTWIUZyFZgk9b6sFJqjVJqobvZPUqpw0qpz4F7gNvd65YAv8b85bAHWONe1mm2HbURGxnGtyYP9N4gZy8UHpGDpkKIXsOncexa6+3A9mbLfuXx+AHggVbWXQes60CNrTpf6eCfpypZPv0y4i0R3hvtWw8RsTDupq4oQYigY7PZ2LZtG7feemub173zzjv54x//SEJCQqttnnjiCaZOncpVV13VkTLFRfToM0/fPJCL0wW3XTnYewO7DQ5tgbGLISq+W2sToqey2Wxs3LjR62v19fUXXfcvf/nLRUMd4N577+1xoe50Ov1dQpv06MnIpw1O4UfTUxneN857g0OvQV01TL69W+sSoif74x//yJkzZ1i0aBFXXXUVM2fO5KmnnqJv375YrVa2b9/OD3/4Q/Lz83E4HNx2223cfPPNAMyaNYvXXnuN6upq7rzzTiZPnsz+/ftJT0/nmWeewWKxcP/99zNz5kzmzp3L9ddfz+LFi3n33XdxOp386U9/YtiwYZSUlPCTn/yEsrIyxo0bx/vvv8/mzZtJSblwDqgHH3yQgwcP4nA4mDNnDvfccw8AX3zxBY888gjV1dVERkayfv16oqOj+cMf/sAHH3wAwNKlS1mxYkVjzSkpKRw8eJDf/e53vPDCCzz55JMUFhZy5swZUlNTWb16NT/96U+pqakB4Je//CWTJk0CzF9oW7duJSQkhGuuuYalS5dy77338vrrrwNw6tQp/vVf/5UtW7Z0y79hjw72cQMTCa+4yN5B9gbzQtUDJrfeRogAtjk7h017z166YStcLhehoRf+Yb50Smbrx6SAn/zkJ3z11Ve8+eabAHz66accPHiQbdu2kZlpDnJ75JFHSEpKwm63c9NNNzF79mySk5MveJ/Tp0/z2GOP8fDDD3Pvvfeyc+dOFi1qeW5jcnIyr7/+Oi+99BLr1q3jP//zP3nqqae44oor+MEPfsDu3bt55ZVXvNa6evVqkpKSqK+v5/bbb+fo0aMMHTqU1atX8/jjj3P55ZdTWVmJxWLhlVdeIScnh9dff53w8HDKysouuf0OHz7Ms88+S3JyMjU1NTz33HNERUVdENT//Oc/efvtt9m0aRPR0dGUlZWRlJREXFxc43DYLVu2cOONN17y8zpLjw72i8r7HPIOwNzfgowwEKJDxo0b1xjqAC+88AK7du0CIC8vj9OnT7cI9oEDBzaO8R8zZgznzp3Dm9mzZwMwduzYxvfMzs7mqaeeAuCaa64hMTHR67o7duxg06ZNOJ1OioqKOH78OCEhIaSlpXH55eYouLg48y/6jz/+mGXLlhEebsZeUpKXi9w3M2vWLCwW82L3TqeTNWvWcPToUUJDQzl16lTj+y5evLhxrHvD+y5ZsoTNmzfzwAMPsH37dl599dVLfl5nCd5gz94AYVFw+VJ/VyJEu31r8sCL7l1fSmedXBMTE9P4+NNPP+Wjjz7ilVdeITo6mhUrVuBwOFqsExkZ2fg4LCzMaxuAiAhz4ENoaGhjH74vZ92ePXuWdevW8dprr5GYmMj999+Pw+HAMAyvwwNbWx4WFtb4ec1r9Nx269evJzU1lTfffBOXy9X4i6O1950zZw5PP/00V1xxBWPGjGnxi68r9eiDp62qrYaDr8LoRRDTyrzsQgivYmNjqaqqavX1iooKEhMTiY6O5vjx4xw4cKDTa5g8eTI7duwA4IMPPqC8vLxFm6qqKqKjo4mPj+f8+fPs3r0bgKFDh1JYWNg4DUBlZSVOp5MZM2bw8ssvNx4IbeiKGTBgAIcOHQLgH//4R6s1VVRUkJaWRmhoKG+++WbjL6EZM2awefPmxr73hveNiori6quv5qGHHmLx4sUd3iZtEZzBfuQNcNhgsoxdF6KtkpOTmTRpEvPnz+e3v/1ti9evueYanE4nCxYs4IknnmDChAmdXsOqVav48MMPufHGG9m9ezdpaWmNXSoNRo0axejRo5k3bx4/+9nPGg9kRkZG8vjjj/Pwww+zcOFC7rjjDhwOB0uWLKF///4sXLiQhQsX8re//a3xsx555BGWL19OWFhYqzUtX76c119/naVLl3Lq1KnGv2KuueYaZs2axbe+9S0WLVrEunVNo7sXLFhASEgIV199dWdvooszDMPvt7179xrtdeTIkZYL1842jD9PMgyXq93v21m81hdApL6O6Yr6OvM9q6urO+29ukJr9TkcDqOurs4wDMPYt2+fsXDhwu4s6wId2YZr1641Hn/88U6pw9v3wp2dLTI1+PrYC4/C2U/gujVy0FSIHio3N5f77rsPl8tFREQEv/71r/1dUpv96Ec/4syZM2zY0GXXFmpV8AX7vuchNBzGL/d3JUKIdho8eDBvvPGGv8vokKefftpvnx1cfexOB3y+EdQ3Ic7L9L1CCNELBFewW7dBTYkcNBVC9GrBFez7NkDiIBg6y9+VCCGE3wRPsJecgJO7YdIKCA2eH0sIIdoqeBJw3wsQEgoT2j7VqBCiYyZONC85WVBQ0DgRV3MrVqzg4MGDF32f9evXN57oA+Y0wDabrfMK7SWCI9jr6+DASzBiNiR6vVa2EKIbpKen8+c//7nd6z///PMXBLsv0wAHEsMwcLlc/i4jSIL9y51QWSBXSRKiE/z+97/npZdeanz+5JNPsm7dOqqqqli5ciU33ngjCxYs4K233mqxbk5ODvPnzwfMa7euXr2aBQsWcN99911wzc4HH3yQxYsXs3jx4sZfBM8//zyFhYWsXLmSFStWAOYkXCUl5kXXnnvuOebPn8/8+fNZv3594+ddf/31/OIXv2DevHnccccdXq8N+s4777BkyRJuuOEGbr/9ds6fPw+Y0xI88MADLFiwgAULFrBzp3nlzt27d3PjjTeycOFCvv/97zduh2effbbxPefPn09OTk5jDQ899BA33ngjeXl5jT/fvHnzLvhF98UXX7Bs2TIWLlzITTfdRGVlJcuXL8dqtTa2WbZsGUePHvX1n8ur4BjHvm8DxPUz99iFCCYHNsL+F9u9eqSrHkKbnSY/8dsw4ZZW15k3bx6PPPJI4xWUduzYwdq1a4mKiuLpp58mLi6OkpISbr75Zq699tpWr8e5ceNGLBYL27Zt4+jRoxfMl9Iw3W5lZSV33303R48e5bbbbmP9+vVs2LChxbzrhw4dYsuWLWzatAnDMFi6dCnTpk0jISHBp+mBJ0+ezKZNmwgJCeHVV19l7dq13H///TzzzDPExcWxbds2AMrLyykpKeGXv/wlL774IpmZmeTn519yO588eZLf/OY3PPTQQxf8fL5MJ7xkyRK2bNnCz3/+c06ePEltbS2jRo265GdeTI8P9vDqAjj2Fly9GsJ6/I8jhN+NHj2a4uJiCgoKKC0tJSEhgYyMDOrq6njsscfYs2cPoaGhFBQUcP78edLSvJ8zsmfPnsY971GjRqGUanytYbrduro6zp8/z/Hjxy8aZtnZ2XzjG99onJ/luuuuY+/evcyaNcun6YHz8/NZvXo1RUVF1NbWMnCgOWPmxx9/zGOPPdbYLjExkXfeeYcpU6Y0TlPc2pTBnjIyMi6YM6ct0wnPnTuXZ555hp/+9Kds3ry5UyYM6/FJmHRiGxgumLjC36UI0fkm3HLRvetLqW3ntL1z5sxh586dnD9/nnnz5gGwbds2SkpK2LJlCxEREcyaNavVqXgbeNub95xuNzIykv/4j/+45PsYF5nG15fpgR9++GFuv/12rr32Wj799NPGud4NL1PuelvW8N6e/eeen+M5rXFbpxOOjo7mqquu4u2332bHjh1s3ry51Z/VVz27j91VT9LJv8HQmZAyxN/VCBE05s2bx/bt29m5cydz5swBzGlr+/TpQ0REBJ988kmrF85oMHXq1MYuji+//BKtNXDhdLvFxcWN0+1C61MGT506lbfeeouamhqqq6t56623mDJlis8/T0VFBenp6QAXTFUwY8YMXnyxqaurvLyciRMnsmfPHs6ePdu4DMzpfY8cOQKYV1bKycnx+lltnU4YzItyPPzww4wbN86nC4BcSs/eYz/+LhHV+TDpN/6uRIigMmLECKqqqujbty99+/YFzClo7777bhYvXkxWVhZDhw696HvccsstjQcms7KyGrsgPKfbzcjIaJxuF8zrkN55552kpaXxwgsvNC4fM2YMixcvZsmSJQDcdNNNjB49utVwbW7VqlXce++9pKenM378+Mb17r77btasWcP8+fMJDQ1l1apVzJ49mzVr1vDjH/8Yl8tFUlISzz//PHPmzOHNN99k0aJFjBs3jsGDB3v9LM+fLzMz0+t0wna7HYvFwnPPPUd4eDhjx44lLi6u8+Zt9zblY3ff2j1t7wd/Mmp/O9Iw6uztW78b9MZpZztTb6xPpu0NLN1RY35+vjF79myjvr6+1TZtmba3Z3fFXHUPx+f+FcKj/F2JEEK0yxtvvMHSpUu57777Wlx4vL16dldMSAhGRMyl2wkhRIC64YYbuOGGGzr1PXv2HrsQQcrw4WLOovdo6/dBgl2IAGOxWCguLpZwF4AZ6sXFxVgsFp/X6dldMUIEoYEDB5KTk0NRUVGH36uuro6IiIhOqKprBHp9EBg1WiyWxpOqfCHBLkSAiYiIYMiQzjkvw2q1Np6VGYgCvT7oGTU251OwK6XmAk8AYcBarfWjrbS7CXgVmKq13quUGgxYAe1u8onW+q4OVy2EEKJVlwx2pVQY8DRwHZAD7FFKbdVaH2nWLh64B/i02Vsc11pPQAghRLfw5eDpNOCY1vqE1roWeBlY5KXdr4HfAS3nzBRCCNFtfOmKGQCc9XieA0z3bKCUmghkaq3/ppT6f83WH6KU2g/YgF9ord/39iHZ2dm+V92J63YHqa9jpL6Okfo6rifU6MmXYPc22XLjOCylVCjwOHC7l3Z5wCCtdbFSajLwhlJqjNb6gmtdTZ482fuEzkIIIdrMl66YHCDT4/lAINfjeTwwFnhPKXUKuALYqpSaorV2aK2LAbTW2cBxYGQn1C2EEKIVvuyx7wFGKKWGAOeAZcDyhhe11uVAasNzpdR7wP9zj4pJA0q01vVKqaHACOBEJ9YvhBCimUsGu9baqZRaBezEHO64Tmt9WCm1Btirtd56kdWvAdYopZxAPXCX1rqkPYVeasilUioKeB6YDBQDN2utT7Xns9pRW6b7s/sBLuB/tdZPNGszE3gTOOletEVrvaY76nN//imgAvPfwam1ntLs9RDM7ftNoBq4XWu9r5tqU8ArHouGAr/SWv/Jo81MunH7KaXWAfOBQq31WPeyFHedg4FTwFKtdamXdVcCv3A/fVhrvaGb6vs9sACoxfzr+Dta6zIv657iIt+FLqzvIeBOoOHMq59prbd7Wden4dVdUN8rQMNlnpKAMm8j+rpj+3WUT+PY3Rt/e7Nlv2ql7UyPx5uBDl8OxMchl98FSrXWw5VSy4DfAjd39LN95AR+orXe5x72ma2U2tV8SCjwvtZ6fjfV5M3XtdbnW3ntesy/qEZgHhz/L5odJO8q2rwCwwRo/Lc+B7zupWl3br/1wFOYv7Ab3A+8rbV+VCl1v/v5v3uu5A7/B4EpmMeist3f1Ra/ALqgvl3AA+6dsd8CDzSvz8PFvgtdVR/A41rrP7S2kq/Dq7uiPq11Y14opf4IlF9k/a7efh3SU+aK8WXI5SKgYc/oNeBa915ol9Na5zXs3WqtKzBPyhrQHZ/diRYBz2utDa31J0CSUqq/H+q4FvPch9N++OxGWuvdQPO/Lj2/YxsAb1PyzQF2aa1L3GG+C5jbHfVprf+htXa6n36CeTzML1rZfr7wdXh1h1ysPnduLAU2dvbndpeeEuzehlw2D87GNu4vdznQp1uq8+A+23YiLU/UArhSKfW5UmqHUmpM91aGAfxDKZWtlPq+l9d92cbdYRmt/4fy5/YDSNda54H5yxzo66VNoGzHO4Adrbx2qe9CV1qllPpCKbVOKZXs5fVA2H5fAwq01l+18ro/t59PekqwX3TIZRvadCmlVBxm19N9zYd0AvuAy7TW44EngTear9/FZmitJ2F2ufxIKXVNs9cDYftFAgsxp6Vozt/bz1eBsB1/jtk9+FIrTS71Xegq/wUMw+x2ywP+6KWN37cfcAsX31v31/bzWU8J9ksNubygjVIqHEikfX8KtotSKgIz1F/SWm9p/rrW2qa1rnQ/3g5EKKVSm7frKlrrXPd9IWb/9bRmTXzZxl3temCf1rqg+Qv+3n5uBQ3dU+77Qi9t/Lod3Qdu5wO3aq29BqIP34UuobUu0FrXa61dwF9a+Vx/b79wYDEXHsy/gL+2X1v0lGBvHHLp3qtbBjQfjbMVWOl+fBPwTmtf7M7m7pN7FrBqrR9rpU2/hj5/pdQ0zG1f3E31xboP6qKUigVmA4eaNdsK3KaUClFKXQGUN3Q7dKNW95T8uf08eH7HVmKO0mluJzBbKZXs7mqY7V7W5dyjSf4dWKi1rm6ljS/fha6qz/OYzY2tfK4v/9e70jeAo1prr1fJ9uf2a4seMW2vj0MunwVeUEodw9xTX9aNJc4AVgAHlVIH3Mt+Bgxy1//fmL9s7nYP/awBlnXXLx4gHXjdHFVIOPBXrfXflVJ3edS3HXOo4zHM4Y7f6abaAFBKxWCOhPiBxzLP+rp1+ymlNgIzgVSlVA7mSJdHgU1Kqe8CZ4Al7rZTMIfyfk9rXaKU+jVmQAGsae8Q33bU9wAQBexy/1t/orW+SymVgTls8Ju08l3opvpmKqUmYHatnML9b+1ZX2v/17ujPq31s3g5xuOP7ddRIXKVFiGECC49pStGCCGEjyTYhRAiyEiwCyFEkJFgF0KIICPBLoQQQUaCXQghgowEuxBCBBkJdiGECDL/HxCtUnlvZdXyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe99fd1a4d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_accs, label='training accuracy')\n",
    "plt.plot(val_accs, label='validation accuracy')\n",
    "plt.legend(loc=0, frameon=True)\n",
    "# plt.xlim(0, 19)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py27]",
   "language": "python",
   "name": "conda-env-py27-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
